RMD environment requirements:

#rStudio 3.2.4 #Internet access to download DropBox hosted csv file [128MB] #Folder ‘sfzipcodes’ located in RMD’s working directory #https://github.com/avennu01/three-amigos/tree/master/sfzipcodes

Load and wrangle the data

## dat_crime <- read.csv(unz("https://github.com/avennu01/three-amigos/blob/master/train.csv.zip","train.csv")) .. unable to direclty read data from GitHub

dat_crime <- read.csv("https://www.dropbox.com/s/kjkt5ndf3jkibq4/train.csv?raw=1")

#table(dat_crime$Category)

# seperating Dates in to Date & Time and selecting below categories only, rest have been eliminated based on group consensus since they dint have impact on our end objective

# Missing Person .. Need more deepdive 

dat_crimeNew <- dat_crime %>% separate(col = Dates, into = c("Date","Time"), sep = " ",fill = "right" ) %>% filter(Category %in% c("ASSAULT","DISORDERLY CONDUCT","DRUNKENNESS","DRUG/NARCOTIC","KIDNAPPING","LARCENY/THEFT","LOITERING","MISSING PERSON","NON-CRIMINAL","ROBBERY","SECONDARY CODES","SEX OFFENSES FORCIBLE","SEX OFFENSES NON FORCIBLE","STOLEN PROPERTY","SUSPICIOUS OCC","VANDALISM","VEHICLE THEFT"))

dat_occ <- dat_crimeNew %>% group_by(Category,Descript) %>% summarize(occurence= n())

# Based of above table, below considerations have been made 
# Descripts to consider apart from one's selected .... ASSAULT, BATTERY,STALKING,LOITERING,
# complete categories to consider "KIDNAPPING","SEX OFFENSES FORCIBLE","SEX OFFENSES NON FORCIBLE"
# complete categories to ingore "VANDALISM","VEHICLE THEFT"

dat_crimeNew <- dat_crime %>% separate(col = Dates, into = c("Date","Time"), sep = " ",fill = "right" ) %>% filter(Category %in% c("ASSAULT","DISORDERLY CONDUCT","DRUNKENNESS","DRUG/NARCOTIC","KIDNAPPING","LARCENY/THEFT","LOITERING","MISSING PERSON","NON-CRIMINAL","ROBBERY","SECONDARY CODES","SEX OFFENSES FORCIBLE","SEX OFFENSES NON FORCIBLE","STOLEN PROPERTY","SUSPICIOUS OCC"))

dat_occ <- dat_crimeNew %>% group_by(Category,Descript) %>% summarize(occurence= n())

p <- dat_occ %>% spread(Descript,occurence)

final_desc <- c("AGGRAVATED ASSAULT WITH","ASSAULT, AGGRAVATED, W/","ATTEMPTED HOMICIDE WITH","ATTEMPTED MAYHEM WITH","ATTEMPTED SIMPLE","BATTERY WITH","MAYHEM WITH","THREATS AGAINST","WILLFUL CRUELTY","COMMITTING PUBLIC","DISTURBING THE PEAC","MAINTAINING A PUBLIC","FOR SALE","SALE OF","UNDER INFLUENCE","UNDER THE INFLUENCE","ATTEMPTED PETTY THEFT","GRAND THEFT PICK","GRAND THEFT PURSE","PETTY THEFT","THEFT, DRUNK ROLL,","AIDED CASE, DOG","AIDED CASE, INJURED","ASSAULT TO ROB WITH ","ATTEMPTED ROBBERY ON THE STREET","ATTEMPTED ROBBERY WITH","ROBBERY ON THE STREET ","ROBBERY, ARMED WITH A ","ROBBERY, BODILY","ASSAULT BY JUVENILE","SHOOTING BY JUVENILE","ANNOY OR MOLEST","STOLEN CELLULAR PHONE","STOLEN ELECTRONICS","SUSPICIOUS A","SUSPICIOUS OCCU","SUSPICIOUS PER","MISSING")

select_desc <- function(x) { 
                   names(p %>% select(contains(x)))[-1]
      }

dat_crimefinal <- full_join(dat_crimeNew %>% filter(Descript %in% unlist(lapply(final_desc,select_desc))),dat_crimeNew %>% filter(Category %in% c("KIDNAPPING","SEX OFFENSES FORCIBLE","SEX OFFENSES NON FORCIBLE")))

# Reduction in observations
#dim(dat_crime)
#dim(dat_crimeNew)
#dim(dat_crimefinal)
paste("Original raw data set was ", nrow(dat_crime), " observances.")
## [1] "Original raw data set was  878049  observances."
paste("Original raw data set was BLA BLA", nrow(dat_crimeNew), " observances.")
## [1] "Original raw data set was BLA BLA 509681  observances."
paste("Lastly, all data not impacting a pedistrian was removed, leaving", nrow(dat_crimefinal), " observances.")
## [1] "Lastly, all data not impacting a pedistrian was removed, leaving 195711  observances."
# Based on the dat_occ_final we can categorise the level of serverity of crime if we want to show our results in such manner 
dat_occ_final <- dat_crimefinal %>% group_by(Category,Descript) %>% summarize(occurence= n())

cat_life <- dat_crimefinal %>% filter(Category %in% c("ASSAULT","KIDNAPPING","MISSING PERSON","SECONDARY CODES","SEX OFFENSES FORCIBLE","SEX OFFENSES NON FORCIBLE","SUSPICIOUS OCC"))
cat_prop <- dat_crimefinal %>% filter(Category %in% c("STOLEN PROPERTY","ROBBERY","LARCENY/THEFT"))
cat_nui <- dat_crimefinal %>% filter(Category %in% c("DISORDERLY CONDUCT","DRUG/NARCOTIC","DRUNKENNESS","NON-CRIMINAL"))

  ##Clean up vectors no longer required
rm(dat_crime)
rm(dat_crimeNew)
rm(dat_occ)

Visualizations

#Assigning longitude and latitude to x and y
x <- dat_crimefinal$X
y <- dat_crimefinal$Y

#Creating base map of San Francisco using ggmap
map_SF <- get_map(location = "San Francisco", zoom = 12)
map <- ggmap(map_SF)

#Inspiration and ideas from the following sources
#https://rpubs.com/nickbearman/r-google-map-making
#http://www.r-bloggers.com/contour-and-density-layers-with-ggmap/
#https://rpubs.com/hegupta/151080

#Heat map using all data across all categories with zip code boundaries
W <- dat_crimefinal
plot<- ggmap(map_SF, extent = "panel", maprange=FALSE) +
   geom_density2d(data = W, aes(x = x, y = y)) +
   stat_density2d(data = W, aes(x = x, y = y,  fill = ..level.., alpha = ..level..),
                  size = 0.01, bins = 16, geom = 'polygon') +
   scale_fill_gradient(low = "blue", high = "red") +
   theme(legend.position = "none")+ ggtitle("All crime categories for wrangled data with zip code boundaries")

#Layer zip codes on above mao

#Zip <- readOGR(".","SFZipCodes")
Zip <- readOGR(dsn=path.expand("./sfzipcodes"), layer = "SFZipCodes")
## OGR data source with driver: ESRI Shapefile 
## Source: "./sfzipcodes", layer: "SFZipCodes"
## with 27 features
## It has 11 fields
Zip <- spTransform(Zip, CRS("+proj=longlat +datum=WGS84"))
Zip <- fortify(Zip)

plot + geom_polygon(aes(x=long, y=lat, group=group), fill='grey', size=.2,color='blue', data=Zip, alpha=0.2)

#Heat map across category deemed as property
x1 <- cat_prop$X
y1 <- cat_prop$Y

ggmap(map_SF, extent = "panel", maprange=FALSE) +
   geom_density2d(data = cat_prop, aes(x = x1, y = y1)) +
   stat_density2d(data = cat_prop, aes(x = x1, y = y1,  fill = ..level.., alpha = ..level..),
                  size = 0.01, bins = 16, geom = 'polygon') +
   scale_fill_gradient(low = "green", high = "black") +
   theme(legend.position = "none")+ ggtitle("Category Property Crimes")

#Heat map across category deemed as life
x2 <- cat_life$X
y2 <- cat_life$Y

ggmap(map_SF, extent = "panel", maprange=FALSE) +
   geom_density2d(data = cat_life, aes(x = x2, y = y2)) +
   stat_density2d(data = cat_life, aes(x = x2, y = y2,  fill = ..level.., alpha = ..level..),
                  size = 0.01, bins = 16, geom = 'polygon') +
   scale_fill_gradient(low = "yellow", high = "blue") +
   theme(legend.position = "none")+ ggtitle("Category Life Crimes")

#Heat map across category deemed as nuisance
x3 <- cat_nui$X
y3 <- cat_nui$Y

ggmap(map_SF, extent = "panel", maprange=FALSE) +
   geom_density2d(data = cat_nui, aes(x = x3, y = y3)) +
   stat_density2d(data = cat_nui, aes(x = x3, y = y3,  fill = ..level.., alpha = ..level..),
                  size = 0.01, bins = 16, geom = 'polygon') +
   scale_fill_gradient(low = "yellow", high = "red") +
   theme(legend.position = "none")+ ggtitle("Category Nuisance Crimes")

#Adding column "class" to dataset using nested if statement with mutate

dat <- dat_crimefinal %>% mutate(class = ifelse(Category %in% c("STOLEN PROPERTY","ROBBERY","LARCENY/THEFT"), "Prop", ifelse(Category %in% c("DISORDERLY CONDUCT","DRUG/NARCOTIC","DRUNKENNESS","NON-CRIMINAL"), "Nui", "Life")))

dat1 <- dat %>% group_by(Category) %>% summarize(occurence = n())

#Histogram by Category
qplot(Category, data = dat, geom = "bar", fill = Category) +
    ggtitle("Crime Categories in San Francisco") +
    theme(axis.text.x = element_blank())+
    xlab("Category") + 
    ylab("Occurence")

#Histogram by district
qplot(dat$PdDistrict, data = dat, geom = "bar", fill = class) +
    scale_x_discrete(label = abbreviate) +
    ggtitle("Crime by district and class in San Francisco") +
    xlab("District") + 
    ylab("Crimes by class")

#Further analysis of Southern District

#Daily average crime by class in southern district
dat2 <- dat %>% filter(PdDistrict == "SOUTHERN") %>% group_by(DayOfWeek, class) %>% summarize(DailyAvg = mean(n()))

ggplot(dat2, aes(x = dat2$DayOfWeek, y = dat2$DailyAvg, fill = factor(dat2$class) )) + geom_bar(stat="identity") + labs(x = "Day of Week", y = "Daily Average", title = "Daily Avg Crime by class for Southern District")

#Time of Day analysis for Southern District
dat3 <- dat %>% filter(PdDistrict=="SOUTHERN")

time <- as.data.frame(table(dat3$Time),stringsAsFactors = FALSE)
time$Hour <- substr(time$Var1, 1, 2)
TimeOfDay <- ggplot(time, aes(x = Hour, Freq))
TimeOfDay + stat_summary(fun.y = sum, geom = "bar", fill = "grey") +
    labs(x = "Time of Day", y = "Occurrences", title ="Southern District crime timings")

San Francisco map with all wrangled crime points

##This section is focused on showing the coverage of the crime data points  

  ##Zoomed map of the wrangled crime points for the whole San Francisco area
map_SF <- get_map(location = "San Francisco, CA", zoom = 13)

ggmap(map_SF) +
    geom_jitter(data = dat_crimefinal, aes(X, Y), alpha=0.2, size=0.5, color="red") 

  ##From: http://www.r-bloggers.com/map-biodiversity-records-with-rgbif-and-ggmap-packages-in-r/

The three choosen paths from our hotel at St. Regis

##2 mile walk # North bound
walking_from_1 <- "125 3rd Street, San Francisco, CA 94103" #St. Regis SF
  walking_to_1 <- "2 Beach St, San Francisco, CA 94133"  #Aquarium of the Bay
  walking_route_1 <- route(walking_from_1, walking_to_1, structure = 'route', 
                           mode = 'walking', alternative = FALSE)
##2 mile walk # West bound
walking_from_2 <- "125 3rd Street, San Francisco, CA 94103" #St. Regis SF
  walking_to_2 <- "Painted Ladies, Steiner Street, San Francisco, CA 94117"  #Painted Ladies
  walking_route_2 <- route(walking_from_2, walking_to_2, structure = 'route', 
                           mode = 'walking', alternative = FALSE)
##~1 mile walk # East bound
walking_from_3 <- "125 3rd Street, San Francisco, CA 94103" #St. Regis SF
  walking_to_3 <- "369 The Embarcadero, San Francisco, CA 94105"  #Cupid's Span
  walking_route_3 <- route(walking_from_3, walking_to_3, structure = 'route', 
                           mode = 'walking', alternative = FALSE)

  ##Need to figure out how to auto-focus the map
map_SF_1 <- get_map(location = "500 California St, San Francisco, CA 94104", zoom = 14)


ggmap(map_SF_1) + 
    geom_path(data = walking_route_1,
        aes(x = lon, y = lat), color="brown", size = 1.5, lineend='round') + 
    geom_path(data = walking_route_2,
        aes(x = lon, y = lat), color="gray", size = 1.5, lineend='round'
    ) + 
    geom_path(data = walking_route_3,
        aes(x = lon, y = lat), color="tan", size = 1.5, lineend='round'
    )

#https://rpubs.com/nickbearman/r-google-map-making
#https://www.google.com/maps/dir/Coit+Tower,+1+Telegraph+Hill+Blvd,+San+Francisco,+CA+94133/Cable+Car+Museum,+Mason+Street,+San+Francisco,+CA/@37.7991066,-122.4139408,16z/data=!3m1!4b1!4m13!4m12!1m5!1m1!1s0x8085808c40000001:0xde85b80121f2dd44!2m2!1d-122.4058222!2d37.8023949!1m5!1m1!1s0x808580f2960c5a5f:0xdfcd6cebc1ae9a35!2m2!1d-122.411488!2d37.7946268

The three paths and their associated neighborhood crime

  ##Count the number of route steps
  ###Take off a count, as the last line the in vector is not a movement point
walking_route_1_hop_count <- nrow(walking_route_1) - 1
walking_route_2_hop_count <- nrow(walking_route_2) - 1
walking_route_3_hop_count <- nrow(walking_route_3) - 1

walking_route_1_end <- walking_route_1[walking_route_1_hop_count]

  ##Find the Max North
route_1_mNorth <- max(walking_route_1$lat[1:(walking_route_1_hop_count)])
route_2_mNorth <- max(walking_route_2$lat)
route_3_mNorth <- max(walking_route_3$lat)
  ##Find the Max East
route_1_mEast <- max(walking_route_1$lon)
route_2_mEast <- max(walking_route_2$lon)
route_3_mEast <- max(walking_route_3$lon)
  ##Find the Max South
route_1_mSouth <- min(walking_route_1$lat[1:(walking_route_1_hop_count)])
route_2_mSouth <- min(walking_route_2$lat[1:(walking_route_2_hop_count)])
route_3_mSouth <- min(walking_route_3$lat[1:(walking_route_3_hop_count)])
  ##Find the Max West
route_1_mWest <- min(walking_route_1$lon)
route_2_mWest <- min(walking_route_2$lon[1:(walking_route_2_hop_count)])
route_3_mWest <- min(walking_route_3$lon)
  
  #Find the crime points from the wrangled that within the neighborhood of the respective walking path/route.
dat_crimefinal_route_1 <- dat_crimefinal %>% filter(Y <= (route_1_mNorth + 0.0001),Y >= (route_1_mSouth - 0.0001),X >= (route_1_mWest - 0.001), X <= (route_1_mEast + 0.001))

dat_crimefinal_route_2 <- dat_crimefinal %>% filter(Y <= (route_2_mNorth + 0.0001),Y >= (route_2_mSouth - 0.0001),X >= (route_2_mWest - 0.001), X <= (route_2_mEast + 0.001))

dat_crimefinal_route_3 <- dat_crimefinal %>% filter(Y <= (route_3_mNorth + 0.0001),Y >= (route_3_mSouth - 0.0001),X >= (route_3_mWest - 0.001), X <= (route_3_mEast + 0.001))


##This just renders the crime within the lat/lon constraints of how the map was centered
ggmap(map_SF_1)  + 

      ## Start with mapping the in-scope crime data points
  geom_jitter(data = dat_crimefinal_route_1, aes(X, Y), alpha=0.3, size=1, color="purple") +
  geom_jitter(data = dat_crimefinal_route_2, aes(X, Y), alpha=0.3, size=1, color="blue") +
  geom_jitter(data = dat_crimefinal_route_3, aes(X, Y), alpha=0.3, size=1, color="orange") +

      ##Add in x and y axis labels
  xlab("Longitude")+ylab("Latitude") +
  
    ## Next, lay out the walking paths for the three routes.
  geom_path(data = walking_route_1,
        aes(x = lon, y = lat), color="brown", size = 1.5, lineend='round') +
  geom_path(data = walking_route_2,
        aes(x = lon, y = lat), color="gray", size = 1.5, lineend='round') +
  geom_path(data = walking_route_3,
        aes(x = lon, y = lat), color="tan", size = 1.5, lineend='round') +
  
    ##Dot & label for Starting Point
  geom_point(data=walking_route_1[1,],  color="white", size=6) +
  geom_label(data=walking_route_1[1,], label="St. Regis", hjust=1.2, vjust=-0.5, size=5 ) +
  
    ##Dots & labels for ending points
  geom_point(data=walking_route_1[walking_route_1_hop_count,],  color="yellow", size=4) +
    geom_label(data=walking_route_1[walking_route_1_hop_count,], label="Aquarium of the Bay", vjust=-1, size=5)+
  geom_point(data=walking_route_2[walking_route_2_hop_count,],  color="yellow", size=4) +
    geom_label(data=walking_route_2[walking_route_2_hop_count,], label="Painted Ladies", vjust=1.5, size=5)+
  geom_point(data=walking_route_3[walking_route_3_hop_count,],  color="yellow", size=4) +
    geom_label(data=walking_route_3[walking_route_3_hop_count,], label="Cupid's Span", vjust=-0.5, size=5)

#dat_crime # All wrangled full data set
#dat_crime_routes # Crime points within the choosen path's neighborhoods

# Crime categories = life[*0.6], nuisance[(0.1], property[*0.3]
  ##Combine all three individual routes into a single data fram
dat_crimefinal_route_1 <- dat_crimefinal_route_1 %>% mutate(path = 1)
dat_crimefinal_route_2 <- dat_crimefinal_route_2 %>% mutate(path = 2)
dat_crimefinal_route_3 <- dat_crimefinal_route_3 %>% mutate(path = 3)
dat_crimefinal_routes <- rbind(dat_crimefinal_route_1, dat_crimefinal_route_2, dat_crimefinal_route_3)
  ## Clean-up variables no longer required
#rm(dat_crime_route_1, dat_crime_route_2, dat_crime_route_3)

 dat_crimefinal_routes <- dat_crimefinal_routes %>% mutate(class = ifelse(Category %in% c("STOLEN PROPERTY","ROBBERY","LARCENY/THEFT"), "Prop", ifelse(Category %in% c("DISORDERLY CONDUCT","DRUG/NARCOTIC","DRUNKENNESS","NON-CRIMINAL"), "Nui", "Life")))

#prop.table(table(dat_crime_routes %>% group_by(path, class)))

#prop.table(table(filter(dat_crime_routes,path ==1)))

#dat_crime_routes %>% group_by(path, class) %>% summarise(n())
#dat_crime_routes %>% group_by(path, class)

route_1_crime_ct_Life <- dat_crimefinal_routes %>% filter(path ==1, class == "Life") %>% nrow() * 0.6
route_1_crime_ct_Nui <- dat_crimefinal_routes %>% filter(path ==1, class == "Nui") %>% nrow() * 0.3
route_1_crime_ct_Prop <- dat_crimefinal_routes %>% filter(path ==1, class == "Prop") %>% nrow() * 0.1
route_1_crime_relative_risk <- route_1_crime_ct_Life + route_1_crime_ct_Nui + route_1_crime_ct_Prop

route_2_crime_ct_Life <- dat_crimefinal_routes %>% filter(path ==2, class == "Life") %>% nrow() * 0.6
route_2_crime_ct_Nui <- dat_crimefinal_routes %>% filter(path ==2, class == "Nui") %>% nrow() * 0.3
route_2_crime_ct_Prop <- dat_crimefinal_routes %>% filter(path ==2, class == "Prop") %>% nrow() * 0.1
route_2_crime_relative_risk <- route_2_crime_ct_Life + route_2_crime_ct_Nui + route_2_crime_ct_Prop

route_3_crime_ct_Life <- dat_crimefinal_routes %>% filter(path ==2, class == "Life") %>% nrow() * 2 *0.6
route_3_crime_ct_Nui <- dat_crimefinal_routes %>% filter(path ==2, class == "Nui") %>% nrow() * 2* 0.3
route_3_crime_ct_Prop <- dat_crimefinal_routes %>% filter(path ==2, class == "Prop") %>% nrow() *2* 0.1
route_3_crime_relative_risk <- route_3_crime_ct_Life + route_3_crime_ct_Nui + route_3_crime_ct_Prop

#route_1_crime_relative_risk
#route_2_crime_relative_risk
#route_3_crime_relative_risk


  ##Route risk ranking, 10 being worst risk
  ##Make this dynamic in the map rendering
route_risk_rank_1 <- c(1, round(route_1_crime_relative_risk/   max(route_1_crime_relative_risk,route_2_crime_relative_risk,route_3_crime_relative_risk) * 10))
route_risk_rank_2 <- c(2, round(route_2_crime_relative_risk/ max(route_1_crime_relative_risk,route_2_crime_relative_risk,route_3_crime_relative_risk) * 10))
route_risk_rank_3 <- c(3, round(route_3_crime_relative_risk / max(route_1_crime_relative_risk,route_2_crime_relative_risk,route_3_crime_relative_risk) * 10))
route_risk_rank <- rbind(route_risk_rank_1, route_risk_rank_2, route_risk_rank_3)
colnames(route_risk_rank) <- c("path", "risk_rank")

Three paths and heat map of the in-scope crime points

W <- dat_crimefinal_route_1
ggmap(map_SF_1, extent = "panel", maprange=FALSE) +
      ##Create heatmap for neighborhood #1
   geom_density2d(data = dat_crimefinal_route_1, aes(x = X, y = Y)) +
   stat_density2d(data = dat_crimefinal_route_1, aes(x = X, y = Y,  fill = ..level.., alpha = ..level..),
                  size = 0.01, geom = 'polygon') + #, bins = 16
   scale_fill_gradient(low = "blue", high = "red") +
   scale_alpha(range = c(0.00, 0.75), guide = FALSE) +
   theme(legend.position = "none", axis.title = element_blank(), text = element_text(size = 12))+
##Create heatmap for neighborhood #2
  geom_density2d(data = dat_crimefinal_route_2, aes(x = X, y = Y)) +
   stat_density2d(data = dat_crimefinal_route_2, aes(x = X, y = Y,  fill = ..level.., alpha = ..level..), size = 0.01, geom = 'polygon') + #, bins = 16
##Create heatmap for neighborhood #3
  geom_density2d(data = dat_crimefinal_route_3, aes(x = X, y = Y)) +
   stat_density2d(data = dat_crimefinal_route_3, aes(x = X, y = Y,  fill = ..level.., alpha = ..level..), size = 0.01, geom = 'polygon') + #, bins = 16
  
      ## Next, lay out the walking paths for the three routes.
  geom_path(data = walking_route_1,
        aes(x = lon, y = lat), color="blue", size = 2.5, lineend='round') +
  geom_path(data = walking_route_2,
        aes(x = lon, y = lat), color="orange", size = 1.5, lineend='round') +
  geom_path(data = walking_route_3,
        aes(x = lon, y = lat), color="red", size = 1.5, lineend='round') +
  
    ##Dot & label for Starting Point
  geom_point(data=walking_route_1[1,],  color="white", size=6) +
  geom_label(data=walking_route_1[1,], label="St. Regis", hjust=0.2, vjust=2.0, size=5 ) +
  
    ##Dots & labels for ending points
  geom_point(data=walking_route_1[walking_route_1_hop_count,],  color="yellow", size=4) +
    geom_label(data=walking_route_1[walking_route_1_hop_count,], label="Aquarium of the Bay", vjust=-1, size=5)+
  geom_point(data=walking_route_2[walking_route_2_hop_count,],  color="yellow", size=4) +
    geom_label(data=walking_route_2[walking_route_2_hop_count,], label="Painted Ladies", vjust=1.5, size=5)+
  geom_point(data=walking_route_3[walking_route_3_hop_count,],  color="yellow", size=4) +
    geom_label(data=walking_route_3[walking_route_3_hop_count,], label="Cupid's Span", vjust=-1, size=5) +
 ##Add in x and y axis labels
  xlab("Longitude")+ylab("Latitude")